package com.mmm.nlp.impl;

import com.mmm.nlp.TextTokenizer;
import com.mmm.nlp.Token;
import org.springframework.stereotype.Component;

import java.util.LinkedList;
import java.util.Queue;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Created with IntelliJ IDEA.
 * User: Zhendong
 * Date: 4/20/13
 * Time: 10:10 AM
 * To change this template use File | Settings | File Templates.
 */
@Component("tokenizer")
public class RETextTokenizer implements TextTokenizer
{
  public static Pattern WORD_PATTERN = Pattern.compile("([a-zA-Z\\-]*)([^a-zA-Z\\-]*)", Pattern.MULTILINE);

  @Override
  public Queue<Token> tokenize(String text)
  {
    Queue<Token> queue = new LinkedList<Token>();
    if(text == null)
    {
      return queue;
    }

    Matcher m = WORD_PATTERN.matcher(text);
    while(!m.hitEnd())
    {
      if(m.find())
      {
        if(!m.group(1).isEmpty())
        {
          queue.add(new Token(m.group(1), true));
        }
        if(!m.group(2).isEmpty())
        {
          queue.add(new Token(m.group(2), false));
        }
      }
    }
    return queue;
  }
}
